import os
import numpy as np
import ray
import re
from pathlib import Path
import time
os.environ["MODIN_ENGINE"] = "ray"  # Modin will use Ray
ray.init(num_cpus=2)

base_dir = r'/var/dwhouse'
structs_file = base_dir + '/structs.txt'
if __name__ == '__main__':
    import modin.pandas as pd
    begin = time.time()
    p = Path(base_dir)
    files_in_dir = list(p.glob('**/*xls*'))
    print("=====file list =====\n", *files_in_dir, sep='\n')
    for f in files_in_dir:
        print("=====begin read file : %(n1)s" %{'n1': f} )
        sheets = pd.ExcelFile(f).sheet_names
        for sheet in sheets:
            _new_file = str(f)[0:str(f).rindex('.')-1] + str(sheet) + '.txt'
            df = pd.read_excel(f, sheet_name=sheet, dtype=str, index_col=None, na_values='')
             # shanchkonghang he konglie
            struct = {'file': str(f), 'sheet': sheet, 'cols': df.columns.tolist()}
            with open(structs_file, 'a+') as fw:
                fw.write(str(struct) + '\n')
                fw.close()
            df.to_csv(_new_file, sep='\t')
    end = time.time()
    print("====读取文件:%(n1)s====" % {'n1': end-begin})
